if [ $# != 2 ]
then
  echo "Usage Help: bash run_distribute.sh [RANK_TABLE_FILE] [DEVICE_RANGE] For Multiple Devices In Single Machine"
  exit 1
fi

check_real_path(){
  if [ "${1:0:1}" == "/" ]; then
    echo "$1"
  else
    echo "$(realpath -m $PWD/$1)"
  fi
}

PATH1=$(check_real_path $1)

ROOT_FOLDER="/home/liupan/WingsofPanda/WP-GeneralPurposeMedicalFoundationModels/wpgpmfm"

DEVICE_RANGE=$2
DEVICE_RANGE_LEN=${#DEVICE_RANGE}
DEVICE_RANGE=${DEVICE_RANGE:1:DEVICE_RANGE_LEN-2}
PREFIX=${DEVICE_RANGE%%","*}
INDEX=${#PREFIX}
START_DEVICE=${DEVICE_RANGE:0:INDEX}
END_DEVICE=${DEVICE_RANGE:INDEX+1:DEVICE_RANGE_LEN-INDEX}

if [ ! -f $PATH1 ]
then
    echo "error: RANK_TABLE_FILE=$PATH1 is not a file"
exit 1
fi

if [[ ! $START_DEVICE =~ ^[0-9]+$ ]]; then
    echo "error: start_device=$START_DEVICE is not a number"
exit 1
fi

if [[ ! $END_DEVICE =~ ^[0-9]+$ ]]; then
    echo "error: end_device=$END_DEVICE is not a number"
exit 1
fi

#ulimit -u unlimited

export RANK_TABLE_FILE=$PATH1
export RANK_SIZE=$(($END_DEVICE - $START_DEVICE))

for((i=${START_DEVICE}; i<${END_DEVICE}; i++))
  do
      export DEVICE_ID=${i}
      export RANK_ID=$((i-START_DEVICE))
      rm -rf ./device$i
      mkdir ./device$i
      env > ./device$i/env.log
      PYTHONPATH=${PYTHONPATH} python3 ${ROOT_FOLDER}/unet3d/main_parallal.py > ./device$i/train.log 2>&1 &
  done

test -e ./device${START_DEVICE}/train.log && tail -f ./device${START_DEVICE}/train.log